"use client" import * as React from 'react' import type { LegacyColumnDef as ColumnDef } from '@tanstack/react-table/legacy' import { Binary, LockKeyhole, Plus, Play, Sparkles } from 'lucide-react' import { Tabs, TabsList, TabsTrigger, TabsContent } from '@open-mercato/ui/primitives/tabs' import { Accordion, AccordionItem, AccordionTrigger, AccordionContent, } from '@open-mercato/ui/primitives/accordion' import { Button } from '@open-mercato/ui/primitives/button' import { Switch } from '@open-mercato/ui/primitives/switch' import { StatusBadge, type StatusMap } from '@open-mercato/ui/primitives/status-badge' import { SegmentedControl, SegmentedControlItem } from '@open-mercato/ui/primitives/segmented-control' import { DataTable } from '@open-mercato/ui/backend/DataTable' import { RowActions } from '@open-mercato/ui/backend/RowActions' import { JsonDisplay } from '@open-mercato/ui/backend/JsonDisplay' import { LoadingMessage } from '@open-mercato/ui/backend/detail' import { EmptyState } from '@open-mercato/ui/primitives/empty-state' import { apiCall, apiCallOrThrow, withScopedApiRequestHeaders } from '@open-mercato/ui/backend/utils/apiCall' import { updateCrud } from '@open-mercato/ui/backend/utils/crud' import { buildOptimisticLockHeader } from '@open-mercato/ui/backend/utils/optimisticLock' import { surfaceRecordConflict } from '@open-mercato/ui/backend/conflicts' import { flash } from '@open-mercato/ui/backend/FlashMessages' import { useGuardedMutation } from '@open-mercato/ui/backend/injection/useGuardedMutation' import { useAppEvent } from '@open-mercato/ui/backend/injection/useAppEvent' import { hasAllFeatures } from '@open-mercato/shared/lib/auth/featureMatch' import { useT, useLocale } from '@open-mercato/shared/lib/i18n/context' import { formatDateTime } from '../../../../components/types' import { OPTIONAL_REQUEST_INIT } from '../../../../components/optionalRequest' import { useCoalescedReload } from '../../../../components/useCoalescedReload' import { evalSuiteOutcomeVariant, evalSuiteStatusVariant, formatPassScore, mapEvalRunRow, type EvalRunRow, } from '../../../../components/evalRunTypes' import { AssertionFormDrawer } from './AssertionFormDrawer' import { EvalCaseDrawer } from './EvalCaseDrawer' import { RunEvaluationDrawer } from './RunEvaluationDrawer' import { EvalResultsDrawer } from './EvalResultsDrawer' import { shortCaseId } from '../../../../components/subjectRef' type EvaluationSection = 'assertions' | 'cases' | 'runs' export type EvaluationTabProps = { agentId: string agentLabel: string active: boolean initialSection?: EvaluationSection } /** Shared with `AssertionFormDrawer` (type-only, erased at compile time). */ export type AssertionRow = { id: string key: string scorerKey: string title: string description: string | null appliesTo: string type: 'deterministic' | 'llm_judge' severity: 'gate' | 'warn' config: Record enabled: boolean /** True when the row was inherited from a wildcard (`appliesTo = *`) query. */ wildcard: boolean updatedAt: string | null } type EvalCaseStatus = 'draft' | 'approved' | 'archived' type EvalCaseStatusFilter = 'all' | EvalCaseStatus type EvalCaseRow = { id: string name: string | null status: EvalCaseStatus processType: string | null createdAt: string | null updatedAt: string | null } const CASE_STATUS_FILTERS: EvalCaseStatusFilter[] = ['all', 'draft', 'approved', 'archived'] const severityVariant: StatusMap<'gate' | 'warn'> = { gate: 'error', warn: 'warning' } const caseStatusTone: StatusMap = { draft: 'info', approved: 'success', archived: 'neutral' } const TYPE_ICON: Record<'deterministic' | 'llm_judge', React.ComponentType<{ className?: string }>> = { deterministic: Binary, llm_judge: Sparkles, } function readString(record: Record, ...keys: string[]): string { for (const key of keys) { const value = record[key] if (typeof value === 'string') return value } return '' } function mapAssertion(item: Record, wildcard: boolean): AssertionRow | null { const id = readString(item, 'id') if (!id) return null const type = readString(item, 'type') === 'llm_judge' ? 'llm_judge' : 'deterministic' const severity = readString(item, 'severity') === 'gate' ? 'gate' : 'warn' const config = (item.config && typeof item.config === 'object') ? (item.config as Record) : {} const key = readString(item, 'key') const descriptionRaw = item.description const enabledRaw = item.enabled return { id, key, scorerKey: readString(item, 'scorer_key', 'scorerKey') || (typeof config.scorer === 'string' ? config.scorer : '') || key, config, title: readString(item, 'title'), description: typeof descriptionRaw === 'string' ? descriptionRaw : null, appliesTo: readString(item, 'applies_to', 'appliesTo') || '*', type, severity, enabled: enabledRaw === undefined ? true : Boolean(enabledRaw), wildcard, updatedAt: readString(item, 'updated_at', 'updatedAt') || null, } } function mapCase(item: Record): EvalCaseRow | null { const id = readString(item, 'id') if (!id) return null const statusRaw = readString(item, 'status') return { id, name: readString(item, 'name') || null, status: statusRaw === 'approved' ? 'approved' : statusRaw === 'archived' ? 'archived' : 'draft', processType: readString(item, 'process_type', 'processType') || null, createdAt: readString(item, 'created_at', 'createdAt') || null, updatedAt: readString(item, 'updated_at', 'updatedAt') || null, } } function SummaryMetric({ label, value, tone }: { label: string; value: string; tone?: 'default' | 'error' }) { return (

{label}

{value}

) } export default function EvaluationTab({ agentId, agentLabel, active, initialSection }: EvaluationTabProps) { const t = useT() const locale = useLocale() const [section, setSection] = React.useState(initialSection ?? 'assertions') const [features, setFeatures] = React.useState<{ canManage: boolean; canRun: boolean } | null>(null) const [assertions, setAssertions] = React.useState([]) const [cases, setCases] = React.useState([]) const [runs, setRuns] = React.useState([]) const [isLoading, setIsLoading] = React.useState(false) const [forbidden, setForbidden] = React.useState(false) const [caseFilter, setCaseFilter] = React.useState('all') const [assertionDrawer, setAssertionDrawer] = React.useState<{ open: boolean; row: AssertionRow | null }>({ open: false, row: null }) const [caseDrawer, setCaseDrawer] = React.useState<{ open: boolean; mode: 'view' | 'create'; id: string | null }>({ open: false, mode: 'view', id: null }) const [runDrawerOpen, setRunDrawerOpen] = React.useState(false) const [resultsRunId, setResultsRunId] = React.useState(null) const loadedRef = React.useRef(false) React.useEffect(() => { if (initialSection) setSection(initialSection) }, [initialSection]) const { runMutation, retryLastMutation } = useGuardedMutation<{ retryLastMutation: () => Promise }>({ contextId: 'agent_orchestrator.agentDetail.evaluation', blockedMessage: t('agent_orchestrator.evalCases.flash.actionError'), }) const loadAssertions = React.useCallback(async () => { const [own, wild] = await Promise.all([ apiCall<{ items?: Array> }>( `/api/agent_orchestrator/eval-assertions?appliesTo=${encodeURIComponent(agentId)}&pageSize=100`, OPTIONAL_REQUEST_INIT, { fallback: { items: [] } }, ), apiCall<{ items?: Array> }>( `/api/agent_orchestrator/eval-assertions?appliesTo=${encodeURIComponent('*')}&pageSize=100`, OPTIONAL_REQUEST_INIT, { fallback: { items: [] } }, ), ]) if (own.status === 403 || wild.status === 403) return { forbidden: true, rows: [] as AssertionRow[] } const byId = new Map() for (const item of Array.isArray(own.result?.items) ? own.result.items : []) { const row = mapAssertion(item, false) if (row) byId.set(row.id, row) } for (const item of Array.isArray(wild.result?.items) ? wild.result.items : []) { const row = mapAssertion(item, true) // A row already present from the agent-scoped query wins (it is not wildcard). if (row && !byId.has(row.id)) byId.set(row.id, row) } return { forbidden: false, rows: Array.from(byId.values()) } }, [agentId]) const loadCases = React.useCallback(async () => { const call = await apiCall<{ items?: Array> }>( `/api/agent_orchestrator/eval-cases?agentDefinitionId=${encodeURIComponent(agentId)}&pageSize=100`, OPTIONAL_REQUEST_INIT, { fallback: { items: [] } }, ) if (call.status === 403) return { forbidden: true, rows: [] as EvalCaseRow[] } const rows = (Array.isArray(call.result?.items) ? call.result.items : []) .map(mapCase) .filter((row): row is EvalCaseRow => !!row) return { forbidden: false, rows } }, [agentId]) const loadRuns = React.useCallback(async () => { const call = await apiCall<{ items?: Array> }>( `/api/agent_orchestrator/eval-runs?agentDefinitionId=${encodeURIComponent(agentId)}&pageSize=100`, OPTIONAL_REQUEST_INIT, { fallback: { items: [] } }, ) if (call.status === 403) return { forbidden: true, rows: [] as EvalRunRow[] } const rows = (Array.isArray(call.result?.items) ? call.result.items : []) .map((item) => mapEvalRunRow(item)) .filter((row): row is EvalRunRow => row !== null) return { forbidden: false, rows } }, [agentId]) const reloadAll = React.useCallback(async (opts?: { silent?: boolean }) => { if (!opts?.silent) setIsLoading(true) const [a, c, r] = await Promise.all([loadAssertions(), loadCases(), loadRuns()]) if (a.forbidden || c.forbidden || r.forbidden) { setForbidden(true) setIsLoading(false) return } setForbidden(false) setAssertions(a.rows) setCases(c.rows) setRuns(r.rows) setIsLoading(false) }, [loadAssertions, loadCases, loadRuns]) // Feature probe — a user with only `agents.view` may open this tab; the list // APIs require `eval.manage`, so without it the sections are replaced by an // inline forbidden note rather than fake zeros or an error page. React.useEffect(() => { if (!active || features) return let cancelled = false void apiCall<{ granted?: unknown }>( '/api/auth/feature-check', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ features: ['agent_orchestrator.eval.manage', 'agent_orchestrator.eval.run'] }), }, { fallback: { granted: [] } }, ).then((call) => { if (cancelled) return const granted = Array.isArray(call.result?.granted) ? call.result.granted.map((entry) => String(entry)) : [] setFeatures({ canManage: hasAllFeatures(['agent_orchestrator.eval.manage'], granted), canRun: hasAllFeatures(['agent_orchestrator.eval.run'], granted), }) }) return () => { cancelled = true } }, [active, features]) // Lazy fetch on first activation. React.useEffect(() => { if (!active || loadedRef.current || !features || !features.canManage) return loadedRef.current = true void reloadAll() }, [active, features, reloadAll]) // Live refresh of the runs list — a suite finishing changes its status/outcome. const coalescedReload = useCoalescedReload( React.useCallback(() => { if (loadedRef.current) void loadRuns().then((result) => { if (!result.forbidden) setRuns(result.rows) }) }, [loadRuns]), ) useAppEvent('agent_orchestrator.eval_suite_run.completed', () => { if (active) coalescedReload() }) const toggleAssertion = React.useCallback(async (row: AssertionRow, next: boolean) => { setAssertions((prev) => prev.map((item) => (item.id === row.id ? { ...item, enabled: next } : item))) try { await runMutation({ operation: () => withScopedApiRequestHeaders( buildOptimisticLockHeader(row.updatedAt), () => updateCrud('agent_orchestrator/eval-assertions', { id: row.id, key: row.key, scorerKey: row.scorerKey, title: row.title, description: row.description ?? undefined, appliesTo: row.appliesTo, type: row.type, severity: row.severity, enabled: next, config: Object.keys(row.config).length ? row.config : undefined, }), ), context: { retryLastMutation }, }) const result = await loadAssertions() if (!result.forbidden) setAssertions(result.rows) } catch (err) { setAssertions((prev) => prev.map((item) => (item.id === row.id ? { ...item, enabled: row.enabled } : item))) if (surfaceRecordConflict(err, t)) return flash(t('agent_orchestrator.evalAssertions.flash.toggleError', 'Could not update the assertion'), 'error') } }, [runMutation, retryLastMutation, loadAssertions, t]) const changeCaseStatus = React.useCallback(async (row: EvalCaseRow, action: 'approve' | 'archive') => { try { await runMutation({ operation: () => withScopedApiRequestHeaders( buildOptimisticLockHeader(row.updatedAt), () => apiCallOrThrow( `/api/agent_orchestrator/eval-cases/${encodeURIComponent(row.id)}/${action}`, { method: 'POST' }, ), ), context: { retryLastMutation }, }) flash( t(action === 'approve' ? 'agent_orchestrator.evalCases.flash.approved' : 'agent_orchestrator.evalCases.flash.archived'), 'success', ) const result = await loadCases() if (!result.forbidden) setCases(result.rows) } catch (err) { if (surfaceRecordConflict(err, t)) return flash(t('agent_orchestrator.evalCases.flash.actionError'), 'error') } }, [runMutation, retryLastMutation, loadCases, t]) const canManage = features?.canManage ?? false const canRun = features?.canRun ?? false const filteredCases = React.useMemo( () => (caseFilter === 'all' ? cases : cases.filter((row) => row.status === caseFilter)), [cases, caseFilter], ) const gateAssertionCount = React.useMemo( () => assertions.filter((row) => row.severity === 'gate' && row.enabled).length, [assertions], ) // Latest scored run for the pass % + the delta versus the run before it. const scoredRuns = React.useMemo( () => runs.filter((row) => row.passScore != null), [runs], ) const latestPass = scoredRuns[0]?.passScore ?? null const baselineDelta = React.useMemo(() => { if (scoredRuns.length < 2 || scoredRuns[0].passScore == null || scoredRuns[1].passScore == null) return null return Math.round((scoredRuns[0].passScore - scoredRuns[1].passScore) * 100) }, [scoredRuns]) const caseColumns = React.useMemo[]>(() => [ { accessorKey: 'name', header: t('agent_orchestrator.evalCases.col.name', 'Name'), cell: ({ row }) => row.original.name ? ( {row.original.name} ) : ( {t('agent_orchestrator.evalCases.col.unnamed', 'Unnamed case')} ), }, { accessorKey: 'status', header: t('agent_orchestrator.evalCases.col.status'), cell: ({ row }) => ( {t(`agent_orchestrator.evalCases.status.${row.original.status}`)} ), }, { accessorKey: 'processType', header: t('agent_orchestrator.evalCases.detail.processType'), cell: ({ row }) => ( {row.original.processType ?? '—'} ), }, { accessorKey: 'id', header: t('agent_orchestrator.evalCases.col.sourceId'), cell: ({ row }) => ( {shortCaseId(row.original.id)} ), }, { accessorKey: 'createdAt', header: t('agent_orchestrator.evalCases.col.created'), cell: ({ row }) => ( {formatDateTime(row.original.createdAt, locale) ?? '—'} ), }, ], [t, locale]) const runColumns = React.useMemo[]>(() => [ { id: 'status', header: t('agent_orchestrator.evalRuns.col.status'), enableSorting: false, cell: ({ row }) => ( {t(`agent_orchestrator.evalRuns.status.${row.original.status}`)} ), }, { id: 'outcome', header: t('agent_orchestrator.evalRuns.col.outcome'), enableSorting: false, cell: ({ row }) => { const outcome = row.original.outcome if (!outcome) return — return ( {t(`agent_orchestrator.evalRuns.outcome.${outcome}`)} ) }, }, { id: 'passScore', header: t('agent_orchestrator.evalRuns.col.passScore'), enableSorting: false, cell: ({ row }) => { const value = formatPassScore(row.original.passScore) return value ? {value} : — }, }, { id: 'cases', header: t('agent_orchestrator.evalRuns.col.cases'), enableSorting: false, cell: ({ row }) => {row.original.caseCount}, }, { id: 'when', header: t('agent_orchestrator.evalRuns.col.when'), enableSorting: false, cell: ({ row }) => ( {formatDateTime(row.original.startedAt ?? row.original.createdAt, locale) ?? '—'} ), }, ], [t, locale]) if (!features) { return } if (forbidden || !canManage) { return (
{t('agent_orchestrator.agentDetail.evaluation.forbidden', "You don't have permission to view evaluations.")}
) } return (
{/* Summary strip — metrics with no data are omitted (never fake zeros). */}
{latestPass != null ? ( ) : null} {baselineDelta != null ? ( = 0 ? '+' : ''}${baselineDelta} ${t('agent_orchestrator.agentDetail.evaluation.summary.points', 'pts')}`} tone={baselineDelta < 0 ? 'error' : 'default'} /> ) : null}
setSection(value as EvaluationSection)} variant="underline"> {t('agent_orchestrator.agentDetail.evaluation.section.assertions', 'Assertions')} {t('agent_orchestrator.agentDetail.evaluation.section.cases', 'Cases')} {t('agent_orchestrator.agentDetail.evaluation.section.runs', 'Runs')}
{isLoading ? ( ) : assertions.length === 0 ? ( ) : ( {assertions.map((row) => { const TypeIcon = TYPE_ICON[row.type] return (
{row.title || row.key} {row.scorerKey} {t(`agent_orchestrator.evalAssertions.severity.${row.severity}`)} {row.wildcard ? ( {t('agent_orchestrator.evalAssertions.form.allAgents', 'All agents')} ) : null} { void toggleAssertion(row, next) }} aria-label={t('agent_orchestrator.evalAssertions.form.enabled')} />
{row.description ? (

{row.description}

) : null}
{t('agent_orchestrator.evalAssertions.list.col.appliesTo')}
{row.appliesTo}
{t('agent_orchestrator.evalAssertions.form.key')}
{row.key}
{Object.keys(row.config).length ? ( ) : null}
) })}
)}
setCaseFilter(value as EvalCaseStatusFilter)}> {CASE_STATUS_FILTERS.map((value) => ( {value === 'all' ? t('agent_orchestrator.evalCases.tab.all') : t(`agent_orchestrator.evalCases.status.${value}`)} ))}
{isLoading ? ( ) : cases.length === 0 ? ( ) : ( columns={caseColumns} data={filteredCases} onRowClick={(row) => setCaseDrawer({ open: true, mode: 'view', id: row.id })} emptyState={t('agent_orchestrator.evalCases.empty.title')} rowActions={(row) => ( setCaseDrawer({ open: true, mode: 'view', id: row.id }), }, ...(row.status === 'draft' ? [{ id: 'approve', label: t('agent_orchestrator.evalCases.actions.approve'), onSelect: () => { void changeCaseStatus(row, 'approve') } }] : []), ...(row.status === 'draft' || row.status === 'approved' ? [{ id: 'archive', label: t('agent_orchestrator.evalCases.actions.archive'), onSelect: () => { void changeCaseStatus(row, 'archive') } }] : []), ]} /> )} /> )}
{canRun ? ( ) : null}
{isLoading ? ( ) : runs.length === 0 ? ( ) : ( columns={runColumns} data={runs} onRowClick={(row) => setResultsRunId(row.id)} /> )}
setAssertionDrawer((prev) => ({ ...prev, open }))} assertion={assertionDrawer.row} agentId={agentId} onSaved={() => { void loadAssertions().then((result) => { if (!result.forbidden) setAssertions(result.rows) }) }} /> setCaseDrawer((prev) => ({ ...prev, open }))} mode={caseDrawer.mode} caseId={caseDrawer.id} agentId={agentId} agentLabel={agentLabel} onChanged={() => { void loadCases().then((result) => { if (!result.forbidden) setCases(result.rows) }) }} /> { void loadRuns().then((result) => { if (!result.forbidden) setRuns(result.rows) }) }} /> { if (!open) setResultsRunId(null) }} suiteRunId={resultsRunId} agentLabel={agentLabel} />
) }